package com.eliansun.crawler.pageparse.shanghai;

import com.eliansun.crawler.pageparse.AbstractPageProcess;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * user: lxc
 * data:2019- 07-02-10
 * 耶购网 www.yegoo.com
 */
public class Yegoo extends AbstractPageProcess {

    /***
     * 爬取逻辑
     *   根据url解析页面
     *     判断该界面是否为列表界面
     *     如果为列表界面获取详情界面的url
     *     如果为详情界面，解析详情界面，获取数据
     */
    int count = 0;

    @Override
    public void process(Page page) {

        //列表界面
        if (page.getUrl().toString().indexOf("newSearch") != -1) {
            List<Selectable> nodes = page.getHtml().xpath("//li[@class=goodsBox]//div[@class=g-img]//a").nodes();
            if (nodes.size() == 0) {
                return;
            }


            //列表界面解析出对应的详情url，放到队列中
            for (Selectable node : nodes) {
                String s = node.links().toString();
                page.addTargetRequest(s);
            }

            //获取下一页放到队列中
            List<String> getTotalPage = page.getHtml().xpath("//div[@class=paging]//a/text()").all();
            if (getTotalPage != null) {
                Integer totalPage = Integer.parseInt(getTotalPage.get(getTotalPage.size() - 2));
                for (int i = 1; i < totalPage; i++) {
                    String url = page.getUrl().toString();
                    page.addTargetRequest(url.replace("pageNo=0", "pageNo=" + (i + 1)));
                }
            }
        } else {
            //详情界面
            Map<String, Object> resMap = new HashMap<>();
            resMap.put("goodsName", page.getHtml().xpath("//div[@class=pd_title]//h2/text()").get());

            Map<String, Object> resultMap = new HashMap<>();
            resultMap.put("spec", "");
            resultMap.put("medicalDeviceNo", "");
            resultMap.put("productCompany", "");
            resultMap.put("price", null);
            resultMap.put("brand", "");
            resultMap.put("productName", resMap.get("goodsName"));
            resultMap.put("executiveStandard", "");
            resultMap.put("nationality", "");
            resultMap.put("store", super.cTerrace);
            resultMap.put("url", page.getUrl());
            resultMap.put("version", "");
            page.putField("result", resultMap);

            count++;
            System.out.println(count);
        }
    }


    Site site = Site.me()
            //设置编码
            .setCharset("utf-8")
            //设置抓取间隔时间
            .setSleepTime(1000 * 3)
            //设置超时时间，单位毫秒
            .setTimeOut(1000 * 3)
            //设置重试间隔时间
            .setRetrySleepTime(1000 * 3)
            //设置重试次数
            .setRetryTimes(3);

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new Yegoo())
                .addUrl("http://www.yegoo.com/goods/newSearch.html?title=@key@&pageNo=0")
                //设置多个线程同时抓取数据
                .thread(1)
                .run();

    }

}
